{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Alphabet(dict):\n",
    "    def __init__(self, start_id=1):\n",
    "        self.fid = start_id\n",
    "    \n",
    "    def add(self, item):\n",
    "        idx = self.get(item, None)\n",
    "        if idx is None:\n",
    "            idx = self.fid\n",
    "            self[item] = idx\n",
    "            self.fid += 1\n",
    "        return idx\n",
    "    \n",
    "    def dump(self, fname):\n",
    "        with open(fname, 'w') as out:\n",
    "            for k in sorted(self.keys()):\n",
    "                out.write(\"{}\\t{}\\n\".format(k, self[k]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_embed(fname):\n",
    "    f = open(fname)\n",
    "    cnt, vocab_size, embed_dim = 0, 0, 0\n",
    "    embed_dict = {}\n",
    "    print 'Load embedding file start!'\n",
    "    for line in f:\n",
    "        cnt += 1\n",
    "        if cnt % 10000 == 0:\n",
    "            print cnt\n",
    "        terms = line.strip().split(' ')\n",
    "        if len(terms) == 2:\n",
    "            vocab_size = int(terms[0])\n",
    "            embed_dim = int(terms[1])\n",
    "        if len(terms) == embed_dim + 1:\n",
    "            embed_dict[terms[0]] = np.array([float(ii) for ii in terms[1:]])\n",
    "    print 'Load embedding file finish!'\n",
    "    return embed_dict, vocab_size, embed_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_embed(embed_file):\n",
    "    alphabet, embed_mat = Alphabet(), []\n",
    "    embed_dict, vocab_size, embed_dim = load_embed(embed_file)\n",
    "    unknown_word_idx = 0\n",
    "    embed_mat.append(np.random.uniform(-0.25, 0.25, embed_dim))\n",
    "    for word in embed_dict:\n",
    "        alphabet.add(word)\n",
    "        embed_mat.append(embed_dict[word])\n",
    "    dummy_word_idx = alphabet.fid\n",
    "    embed_mat.append(np.zeros(embed_dim))\n",
    "    return alphabet, embed_mat, unknown_word_idx, dummy_word_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load embedding file start!\n",
      "10000\n",
      "Load embedding file finish!\n",
      "Load embedding file start!\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "220000\n",
      "230000\n",
      "240000\n",
      "250000\n",
      "260000\n",
      "270000\n",
      "280000\n",
      "290000\n",
      "300000\n",
      "310000\n",
      "320000\n",
      "330000\n",
      "340000\n",
      "350000\n",
      "360000\n",
      "370000\n",
      "380000\n",
      "390000\n",
      "400000\n",
      "410000\n",
      "Load embedding file finish!\n"
     ]
    }
   ],
   "source": [
    "char_embed_file = '../ieee_zhihu_cup/char_embedding.txt'\n",
    "word_embed_file = '../ieee_zhihu_cup/word_embedding.txt'\n",
    "char_alphabet, char_embed_mat, unknown_char_idx, dummy_char_idx = parse_embed(char_embed_file)\n",
    "word_alphabet, word_embed_mat, unknown_word_idx, dummy_word_idx = parse_embed(word_embed_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_question(fname, split=False, rate=None):\n",
    "    f = open(fname)\n",
    "    cnt, idx, title_char, title_word, desc_char, desc_word = 0, [], [], [], [], []\n",
    "    print 'Load question file start!'\n",
    "    for line in f:\n",
    "        cnt += 1\n",
    "        if cnt % 10000 == 0:\n",
    "            print cnt\n",
    "        terms = line.strip().split('\\t')\n",
    "        idx.append(terms[0])\n",
    "        if len(terms) == 5:\n",
    "            title_char.append(terms[1])\n",
    "            title_word.append(terms[2])\n",
    "            desc_char.append(terms[3])\n",
    "            desc_word.append(terms[4])\n",
    "        elif len(terms) == 4:\n",
    "            title_char.append(terms[1])\n",
    "            title_word.append(terms[2])\n",
    "            desc_char.append(terms[3])\n",
    "            desc_word.append('')\n",
    "        elif len(terms) == 3:\n",
    "            title_char.append(terms[1])\n",
    "            title_word.append(terms[2])\n",
    "            desc_char.append('')\n",
    "            desc_word.append('')\n",
    "        elif len(terms) == 1:\n",
    "            title_char.append('')\n",
    "            title_word.append('')\n",
    "            desc_char.append('')\n",
    "            desc_word.append('')\n",
    "    print 'Load question file finish!'\n",
    "    if split:\n",
    "        ids = np.arange(cnt)\n",
    "        np.random.seed(1024)\n",
    "        np.random.shuffle(ids)\n",
    "        train_id = ids[:int(cnt*rate)]\n",
    "        val_id = ids[int(cnt*rate):]\n",
    "        print \"Finished\"\n",
    "        return [idx[i] for i in train_id], [title_char[i] for i in train_id], \\\n",
    "                [title_word[i] for i in train_id], [desc_char[i] for i in train_id], \\\n",
    "                [desc_word[i] for i in train_id], \\\n",
    "                [idx[i] for i in val_id], [title_char[i] for i in val_id], \\\n",
    "                [title_word[i] for i in val_id], [desc_char[i] for i in val_id], \\\n",
    "                [desc_word[i] for i in val_id]\n",
    "    print \"Finished\"\n",
    "    return idx, title_char, title_word, desc_char, desc_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def convert2indices(data, alphabet, unknown_word_idx, dummy_word_idx, max_length):\n",
    "    data_idx = []\n",
    "    for item in data:\n",
    "        item_arr = [ii for ii in item.split(',') if ii != '']\n",
    "        ex = np.ones(max_length) * dummy_word_idx\n",
    "        for i, word in enumerate(item_arr):\n",
    "            if i >= max_length:\n",
    "                break\n",
    "            idx = alphabet.get(word, unknown_word_idx)\n",
    "            ex[i] = idx\n",
    "        data_idx.append(ex)\n",
    "    data_idx = np.array(data_idx).astype('int32')\n",
    "    return data_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load question file start!\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "220000\n",
      "230000\n",
      "240000\n",
      "250000\n",
      "260000\n",
      "270000\n",
      "280000\n",
      "290000\n",
      "300000\n",
      "310000\n",
      "320000\n",
      "330000\n",
      "340000\n",
      "350000\n",
      "360000\n",
      "370000\n",
      "380000\n",
      "390000\n",
      "400000\n",
      "410000\n",
      "420000\n",
      "430000\n",
      "440000\n",
      "450000\n",
      "460000\n",
      "470000\n",
      "480000\n",
      "490000\n",
      "500000\n",
      "510000\n",
      "520000\n",
      "530000\n",
      "540000\n",
      "550000\n",
      "560000\n",
      "570000\n",
      "580000\n",
      "590000\n",
      "600000\n",
      "610000\n",
      "620000\n",
      "630000\n",
      "640000\n",
      "650000\n",
      "660000\n",
      "670000\n",
      "680000\n",
      "690000\n",
      "700000\n",
      "710000\n",
      "720000\n",
      "730000\n",
      "740000\n",
      "750000\n",
      "760000\n",
      "770000\n",
      "780000\n",
      "790000\n",
      "800000\n",
      "810000\n",
      "820000\n",
      "830000\n",
      "840000\n",
      "850000\n",
      "860000\n",
      "870000\n",
      "880000\n",
      "890000\n",
      "900000\n",
      "910000\n",
      "920000\n",
      "930000\n",
      "940000\n",
      "950000\n",
      "960000\n",
      "970000\n",
      "980000\n",
      "990000\n",
      "1000000\n",
      "1010000\n",
      "1020000\n",
      "1030000\n",
      "1040000\n",
      "1050000\n",
      "1060000\n",
      "1070000\n",
      "1080000\n",
      "1090000\n",
      "1100000\n",
      "1110000\n",
      "1120000\n",
      "1130000\n",
      "1140000\n",
      "1150000\n",
      "1160000\n",
      "1170000\n",
      "1180000\n",
      "1190000\n",
      "1200000\n",
      "1210000\n",
      "1220000\n",
      "1230000\n",
      "1240000\n",
      "1250000\n",
      "1260000\n",
      "1270000\n",
      "1280000\n",
      "1290000\n",
      "1300000\n",
      "1310000\n",
      "1320000\n",
      "1330000\n",
      "1340000\n",
      "1350000\n",
      "1360000\n",
      "1370000\n",
      "1380000\n",
      "1390000\n",
      "1400000\n",
      "1410000\n",
      "1420000\n",
      "1430000\n",
      "1440000\n",
      "1450000\n",
      "1460000\n",
      "1470000\n",
      "1480000\n",
      "1490000\n",
      "1500000\n",
      "1510000\n",
      "1520000\n",
      "1530000\n",
      "1540000\n",
      "1550000\n",
      "1560000\n",
      "1570000\n",
      "1580000\n",
      "1590000\n",
      "1600000\n",
      "1610000\n",
      "1620000\n",
      "1630000\n",
      "1640000\n",
      "1650000\n",
      "1660000\n",
      "1670000\n",
      "1680000\n",
      "1690000\n",
      "1700000\n",
      "1710000\n",
      "1720000\n",
      "1730000\n",
      "1740000\n",
      "1750000\n",
      "1760000\n",
      "1770000\n",
      "1780000\n",
      "1790000\n",
      "1800000\n",
      "1810000\n",
      "1820000\n",
      "1830000\n",
      "1840000\n",
      "1850000\n",
      "1860000\n",
      "1870000\n",
      "1880000\n",
      "1890000\n",
      "1900000\n",
      "1910000\n",
      "1920000\n",
      "1930000\n",
      "1940000\n",
      "1950000\n",
      "1960000\n",
      "1970000\n",
      "1980000\n",
      "1990000\n",
      "2000000\n",
      "2010000\n",
      "2020000\n",
      "2030000\n",
      "2040000\n",
      "2050000\n",
      "2060000\n",
      "2070000\n",
      "2080000\n",
      "2090000\n",
      "2100000\n",
      "2110000\n",
      "2120000\n",
      "2130000\n",
      "2140000\n",
      "2150000\n",
      "2160000\n",
      "2170000\n",
      "2180000\n",
      "2190000\n",
      "2200000\n",
      "2210000\n",
      "2220000\n",
      "2230000\n",
      "2240000\n",
      "2250000\n",
      "2260000\n",
      "2270000\n",
      "2280000\n",
      "2290000\n",
      "2300000\n",
      "2310000\n",
      "2320000\n",
      "2330000\n",
      "2340000\n",
      "2350000\n",
      "2360000\n",
      "2370000\n",
      "2380000\n",
      "2390000\n",
      "2400000\n",
      "2410000\n",
      "2420000\n",
      "2430000\n",
      "2440000\n",
      "2450000\n",
      "2460000\n",
      "2470000\n",
      "2480000\n",
      "2490000\n",
      "2500000\n",
      "2510000\n",
      "2520000\n",
      "2530000\n",
      "2540000\n",
      "2550000\n",
      "2560000\n",
      "2570000\n",
      "2580000\n",
      "2590000\n",
      "2600000\n",
      "2610000\n",
      "2620000\n",
      "2630000\n",
      "2640000\n",
      "2650000\n",
      "2660000\n",
      "2670000\n",
      "2680000\n",
      "2690000\n",
      "2700000\n",
      "2710000\n",
      "2720000\n",
      "2730000\n",
      "2740000\n",
      "2750000\n",
      "2760000\n",
      "2770000\n",
      "2780000\n",
      "2790000\n",
      "2800000\n",
      "2810000\n",
      "2820000\n",
      "2830000\n",
      "2840000\n",
      "2850000\n",
      "2860000\n",
      "2870000\n",
      "2880000\n",
      "2890000\n",
      "2900000\n",
      "2910000\n",
      "2920000\n",
      "2930000\n",
      "2940000\n",
      "2950000\n",
      "2960000\n",
      "2970000\n",
      "2980000\n",
      "2990000\n",
      "Load question file finish!\n",
      "Finished\n",
      "Load question file start!\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "220000\n",
      "230000\n",
      "240000\n",
      "250000\n",
      "260000\n",
      "270000\n",
      "280000\n",
      "290000\n",
      "300000\n",
      "310000\n",
      "320000\n",
      "330000\n",
      "340000\n",
      "350000\n",
      "360000\n",
      "370000\n",
      "380000\n",
      "390000\n",
      "400000\n",
      "410000\n",
      "420000\n",
      "430000\n",
      "440000\n",
      "450000\n",
      "460000\n",
      "470000\n",
      "480000\n",
      "490000\n",
      "500000\n",
      "510000\n",
      "520000\n",
      "530000\n",
      "540000\n",
      "550000\n",
      "560000\n",
      "570000\n",
      "580000\n",
      "590000\n",
      "600000\n",
      "610000\n",
      "620000\n",
      "630000\n",
      "640000\n",
      "650000\n",
      "660000\n",
      "670000\n",
      "680000\n",
      "690000\n",
      "700000\n",
      "710000\n",
      "720000\n",
      "730000\n",
      "740000\n",
      "750000\n",
      "760000\n",
      "770000\n",
      "780000\n",
      "790000\n",
      "800000\n",
      "810000\n",
      "820000\n",
      "830000\n",
      "840000\n",
      "850000\n",
      "860000\n",
      "870000\n",
      "880000\n",
      "890000\n",
      "900000\n",
      "910000\n",
      "920000\n",
      "930000\n",
      "940000\n",
      "950000\n",
      "960000\n",
      "970000\n",
      "980000\n",
      "990000\n",
      "1000000\n",
      "1010000\n",
      "1020000\n",
      "1030000\n",
      "1040000\n",
      "1050000\n",
      "1060000\n",
      "1070000\n",
      "1080000\n",
      "1090000\n",
      "1100000\n",
      "1110000\n",
      "1120000\n",
      "1130000\n",
      "1140000\n",
      "1150000\n",
      "1160000\n",
      "1170000\n",
      "1180000\n",
      "1190000\n",
      "1200000\n",
      "1210000\n",
      "1220000\n",
      "1230000\n",
      "1240000\n",
      "1250000\n",
      "1260000\n",
      "1270000\n",
      "1280000\n",
      "1290000\n",
      "1300000\n",
      "1310000\n",
      "1320000\n",
      "1330000\n",
      "1340000\n",
      "1350000\n",
      "1360000\n",
      "1370000\n",
      "1380000\n",
      "1390000\n",
      "1400000\n",
      "1410000\n",
      "1420000\n",
      "1430000\n",
      "1440000\n",
      "1450000\n",
      "1460000\n",
      "1470000\n",
      "1480000\n",
      "1490000\n",
      "1500000\n",
      "1510000\n",
      "1520000\n",
      "1530000\n",
      "1540000\n",
      "1550000\n",
      "1560000\n",
      "1570000\n",
      "1580000\n",
      "1590000\n",
      "1600000\n",
      "1610000\n",
      "1620000\n",
      "1630000\n",
      "1640000\n",
      "1650000\n",
      "1660000\n",
      "1670000\n",
      "1680000\n",
      "1690000\n",
      "1700000\n",
      "1710000\n",
      "1720000\n",
      "1730000\n",
      "1740000\n",
      "1750000\n",
      "1760000\n",
      "1770000\n",
      "1780000\n",
      "1790000\n",
      "1800000\n",
      "1810000\n",
      "1820000\n",
      "1830000\n",
      "1840000\n",
      "1850000\n",
      "1860000\n",
      "1870000\n",
      "1880000\n",
      "1890000\n",
      "1900000\n",
      "1910000\n",
      "1920000\n",
      "1930000\n",
      "1940000\n",
      "1950000\n",
      "1960000\n",
      "1970000\n",
      "1980000\n",
      "1990000\n",
      "2000000\n",
      "2010000\n",
      "2020000\n",
      "2030000\n",
      "2040000\n",
      "2050000\n",
      "2060000\n",
      "2070000\n",
      "2080000\n",
      "2090000\n",
      "2100000\n",
      "2110000\n",
      "2120000\n",
      "2130000\n",
      "2140000\n",
      "2150000\n",
      "2160000\n",
      "2170000\n",
      "2180000\n",
      "2190000\n",
      "2200000\n",
      "2210000\n",
      "2220000\n",
      "2230000\n",
      "2240000\n",
      "2250000\n",
      "2260000\n",
      "2270000\n",
      "2280000\n",
      "2290000\n",
      "2300000\n",
      "2310000\n",
      "2320000\n",
      "2330000\n",
      "2340000\n",
      "2350000\n",
      "2360000\n",
      "2370000\n",
      "2380000\n",
      "2390000\n",
      "2400000\n",
      "2410000\n",
      "2420000\n",
      "2430000\n",
      "2440000\n",
      "2450000\n",
      "2460000\n",
      "2470000\n",
      "2480000\n",
      "2490000\n",
      "2500000\n",
      "2510000\n",
      "2520000\n",
      "2530000\n",
      "2540000\n",
      "2550000\n",
      "2560000\n",
      "2570000\n",
      "2580000\n",
      "2590000\n",
      "2600000\n",
      "2610000\n",
      "2620000\n",
      "2630000\n",
      "2640000\n",
      "2650000\n",
      "2660000\n",
      "2670000\n",
      "2680000\n",
      "2690000\n",
      "2700000\n",
      "2710000\n",
      "2720000\n",
      "2730000\n",
      "2740000\n",
      "2750000\n",
      "2760000\n",
      "2770000\n",
      "2780000\n",
      "2790000\n",
      "2800000\n",
      "2810000\n",
      "2820000\n",
      "2830000\n",
      "2840000\n",
      "2850000\n",
      "2860000\n",
      "2870000\n",
      "2880000\n",
      "2890000\n",
      "2900000\n",
      "2910000\n",
      "2920000\n",
      "2930000\n",
      "2940000\n",
      "2950000\n",
      "2960000\n",
      "2970000\n",
      "2980000\n",
      "2990000\n",
      "Load question file finish!\n",
      "Finished\n",
      "Load question file start!\n",
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n",
      "140000\n",
      "150000\n",
      "160000\n",
      "170000\n",
      "180000\n",
      "190000\n",
      "200000\n",
      "210000\n",
      "Load question file finish!\n",
      "Finished\n"
     ]
    }
   ],
   "source": [
    "question_train_file = '../ieee_zhihu_cup/question_train_set.txt'\n",
    "question_test_file = '../ieee_zhihu_cup/question_eval_set.txt'\n",
    "train_idx, train_title_char, train_title_word, train_desc_char, train_desc_word, \\\n",
    "val_idx, val_title_char, val_title_word, val_desc_char, val_desc_word \\\n",
    "            = load_question(question_train_file, split=True, rate=0.9)\n",
    "train_idx_all, train_title_char_all, train_title_word_all, train_desc_char_all, \\\n",
    "                train_desc_word_all = load_question(question_train_file)\n",
    "test_idx, test_title_char, test_title_word, test_desc_char, test_desc_word \\\n",
    "                           = load_question(question_test_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "title_char_max_length = 85#22\n",
    "title_word_max_length = 50#30#13\n",
    "desc_char_max_length = 300#117\n",
    "desc_word_max_length = 150#120#58"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_title_char_indices = convert2indices(train_title_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           title_char_max_length)\n",
    "train_title_word_indices = convert2indices(train_title_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          title_word_max_length)\n",
    "train_desc_char_indices = convert2indices(train_desc_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           desc_char_max_length)\n",
    "train_desc_word_indices = convert2indices(train_desc_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          desc_word_max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_title_char_indices = convert2indices(val_title_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           title_char_max_length)\n",
    "val_title_word_indices = convert2indices(val_title_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          title_word_max_length)\n",
    "val_desc_char_indices = convert2indices(val_desc_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           desc_char_max_length)\n",
    "val_desc_word_indices = convert2indices(val_desc_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          desc_word_max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save train data finished!\n",
      "Save val data finished!\n"
     ]
    }
   ],
   "source": [
    "basedir = './embed'\n",
    "np.save('{}/char_embed_mat.npy'.format(basedir), char_embed_mat)\n",
    "del char_embed_mat\n",
    "np.save('{}/word_embed_mat.npy'.format(basedir), word_embed_mat)\n",
    "del word_embed_mat\n",
    "print 'Save embedding finished!'\n",
    "basedir = './train'\n",
    "np.save('{}/train_idx.npy'.format(basedir), train_idx)\n",
    "del train_idx\n",
    "np.save('{}/train_title_char_indices_2.npy'.format(basedir), train_title_char_indices)\n",
    "del train_title_char_indices\n",
    "np.save('{}/train_title_word_indices_2.npy'.format(basedir), train_title_word_indices)\n",
    "del train_title_word_indices\n",
    "np.save('{}/train_desc_char_indices_2.npy'.format(basedir), train_desc_char_indices)\n",
    "del train_desc_char_indices\n",
    "np.save('{}/train_desc_word_indices_2.npy'.format(basedir), train_desc_word_indices)\n",
    "del train_desc_word_indices\n",
    "print 'Save train data finished!'\n",
    "basedir = './val'\n",
    "np.save('{}/val_idx.npy'.format(basedir), val_idx)\n",
    "del val_idx\n",
    "np.save('{}/val_title_char_indices_2.npy'.format(basedir), val_title_char_indices)\n",
    "del val_title_char_indices\n",
    "np.save('{}/val_title_word_indices_2.npy'.format(basedir), val_title_word_indices)\n",
    "del val_title_word_indices\n",
    "np.save('{}/val_desc_char_indices_2.npy'.format(basedir), val_desc_char_indices)\n",
    "del val_desc_char_indices\n",
    "np.save('{}/val_desc_word_indices_2.npy'.format(basedir), val_desc_word_indices)\n",
    "del val_desc_word_indices\n",
    "print 'Save val data finished!'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_title_char_indices_all = convert2indices(train_title_char_all, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           title_char_max_length)\n",
    "train_title_word_indices_all = convert2indices(train_title_word_all, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          title_word_max_length)\n",
    "train_desc_char_indices_all = convert2indices(train_desc_char_all, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           desc_char_max_length)\n",
    "train_desc_word_indices_all = convert2indices(train_desc_word_all, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          desc_word_max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_title_char_indices = convert2indices(test_title_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           title_char_max_length)\n",
    "test_title_word_indices = convert2indices(test_title_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          title_word_max_length)\n",
    "test_desc_char_indices = convert2indices(test_desc_char, char_alphabet, \\\n",
    "                                           unknown_char_idx, dummy_char_idx, \\\n",
    "                                           desc_char_max_length)\n",
    "test_desc_word_indices = convert2indices(test_desc_word, word_alphabet, \\\n",
    "                                          unknown_word_idx, dummy_word_idx, \\\n",
    "                                          desc_word_max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save train all data finished!\n",
      "Save test data finished!\n"
     ]
    }
   ],
   "source": [
    "basedir = './train_all'\n",
    "np.save('{}/train_idx_all.npy'.format(basedir), train_idx_all)\n",
    "np.save('{}/train_title_char_indices_all_2.npy'.format(basedir), train_title_char_indices_all)\n",
    "np.save('{}/train_title_word_indices_all_2.npy'.format(basedir), train_title_word_indices_all)\n",
    "np.save('{}/train_desc_char_indices_all_2.npy'.format(basedir), train_desc_char_indices_all)\n",
    "np.save('{}/train_desc_word_indices_all_2.npy'.format(basedir), train_desc_word_indices_all)\n",
    "print 'Save train all data finished!'\n",
    "basedir = './test'\n",
    "np.save('{}/test_idx.npy'.format(basedir), test_idx)\n",
    "np.save('{}/test_title_char_indices_2.npy'.format(basedir), test_title_char_indices)\n",
    "np.save('{}/test_title_word_indices_2.npy'.format(basedir), test_title_word_indices)\n",
    "np.save('{}/test_desc_char_indices_2.npy'.format(basedir), test_desc_char_indices)\n",
    "np.save('{}/test_desc_word_indices_2.npy'.format(basedir), test_desc_word_indices)\n",
    "print 'Save test data finished!'"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
